package crawler.proxy.impl;

import crawler.proxy.Proxy;
import crawler.proxy.ProxySpider;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

/**
 * USER: lintc 【lintiancong@zhuojianchina.com】
 * DATE: 2017-09-14 10:42
 */
public class ZhiHuProxy extends ProxySpider{

    public ZhiHuProxy() {
        super("http://lab.crossincode.com/proxy/get/");
    }

    @Override
    public Set<Proxy> getList(int index) {
        Set<Proxy> list = new HashSet<>();
//        catchUrl = catchUrl.replace("index", index + "");
        Connection.Response res = null;
        try {
            res = Jsoup.connect(catchInvokeUrl).header("Accept", "*/*").header("Accept-Encoding", "gzip, deflate")
                    .header("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
                    .header("Content-Type", "application/json;charset=UTF-8")
                    .header("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:48.0) Gecko/20100101 Firefox/48.0").
                            ignoreContentType(true).execute();
            JSONObject obj = JSONObject.fromObject(res.body());
            JSONArray ary = obj.optJSONArray("proxies");
            Proxy proxy;
            for (Object o : ary) {
                JSONObject ob = (JSONObject) o;
                proxy = new Proxy();
                proxy.setIp(ob.optString("http").split(":")[0]);
                proxy.setPort(ob.optString("http").split(":")[1]);
                list.add(proxy);
            }
        } catch (IOException e) {
            LOG.error(catchInvokeUrl + ": 抓取失败 (" + e.getMessage() + ")");
        }
        return list;
    }
}
